package it.java.giw.dataextraction.aleph.crawler;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.LinkedList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

import org.eclipse.jetty.util.ConcurrentHashSet;

public class CrawlerMaster {

	private LinkedBlockingDeque<String> seedList;
	private ConcurrentHashSet<Integer> alreadyVisited;
	private int numworkers;
	private ExecutorService workerPool;
	private Lock lock; 
	private LinkedBlockingQueue<String> workInProgress;

	public CrawlerMaster(int numworkers, LinkedBlockingDeque<String> seedList) {
		this.seedList = seedList;
		this.alreadyVisited = new ConcurrentHashSet<Integer>();
		this.numworkers = numworkers;
		this.workerPool = Executors.newFixedThreadPool(numworkers);
		this.workInProgress = new LinkedBlockingQueue<String>();
		this.lock = new ReentrantLock();

	}

	public void start() {
		CountDownLatch startSignal = new CountDownLatch(1);
		CountDownLatch doneSignal = new CountDownLatch(numworkers);
		AtomicInteger id = new AtomicInteger(0);

		for(int i=0; i<numworkers;i++){
			//richiedo all'executor service di avviare un task per ogni worker thread disponibile
			workerPool.execute(new CrawlerWorker(seedList, workInProgress, lock, alreadyVisited, id, startSignal, doneSignal));
		}
		startSignal.countDown();// let all threads proceed
		try {
			doneSignal.await();// wait for all to finish
			workerPool.shutdownNow();

			final File folder = new File("/home/roberto/workspace/AlephDataExtractor");
			LinkedList<String> indexFiles = new LinkedList<String>();
			indexFiles.addAll(listFilesForFolder(folder));
			try {
				File index = new File("id2url.txt");
				// if file doesnt exists, then create it
				if (!index.exists()) {
					index.createNewFile();
				}
				FileWriter fw = new FileWriter(index.getAbsoluteFile(), true);
				BufferedWriter bw = new BufferedWriter(fw);
				BufferedReader br = null;
				for(String file : indexFiles){
					String sCurrentLine;
					br = new BufferedReader(new FileReader(folder+"/"+file));
					while ((sCurrentLine = br.readLine()) != null) {
						bw.write(sCurrentLine+"\n");
					}
				}
				br.close();
				bw.close();
			}catch (IOException e) {
				e.printStackTrace();
			} 

		} catch (InterruptedException e) {
			workerPool.shutdownNow();
		}


	}

	private static LinkedBlockingQueue<String> listFilesForFolder(final File folder) {
		LinkedBlockingQueue<String> output = new LinkedBlockingQueue<String>();
		for (final File fileEntry : folder.listFiles()) {
			if (fileEntry.isDirectory()) {
				listFilesForFolder(fileEntry);
			} else {
				if(fileEntry.getName().contains("id2url"))
					output.add(fileEntry.getName());
			}
		}
		return output;
	}

}
